# coding=utf-8

import threading
from datetime import datetime
from time import sleep

from difflib import SequenceMatcher


class Info:

    def __init__(self, key, count):
        self.key = key
        self.count = count
        
        self.detail = [key]

    def __str__(self):
        ret = "%s: %s" % (self.count, self.key)
        for str1 in self.detail:
            ret += ("\n    %s" % str1)
        ret += "\n"
        return ret 


class InfoList:

    def __init__(self, index):
        self.index = index
        self.list = []


class GlobalData:

    def __init__(self, count):
        self.count = count
        
        self.all_list = [InfoList(i) for i in range(count)]
        
        self.flag_merge = 1
        
        self.ready_list = []
        self.read_list_count = 0

        self.last_index = 999
        
    def __str__(self):
        ret = "%s: %s" % (self.count, self.key)
        for str1 in self.list:
            ret += ("\n    %s" % str1)
        ret += "\n"
        return ret 

    def isFinish(self):
        return self.flag_merge == count

    def getProcess(self):
        return "info: flag_merge=%s len(ready_list)=%s" % (self.flag_merge, len(self.ready_list))

    def appendReady(self, obj):
        self.ready_list.append(obj)
        self.read_list_count += 1
        
    def appendReadyWithFlag(self, obj):
        self.appendReady(obj)
        self.flag_merge += 1

    def popTop2Ready(self):
        if len(self.ready_list) < 2:
            return False, None, None
        
        self.read_list_count -= 2
        index0 = self.ready_list.pop(0)
        index1 = self.ready_list.pop(0)
        
        infoList1 = self.all_list[index0]
        infoList2 = self.all_list[index1]
        
        if len(infoList1.list) >= len(infoList2.list):
            self.last_index = index0
            return True, infoList1, infoList2
        else:
            self.last_index = index1
            return True, infoList2, infoList1
        
    def getLastInfoList(self):
        print("last_index=%s" % self.last_index)
        return self.all_list[self.last_index].list

    
def p(str1):
    if 0:
        print("%s" % str1)


def isSimilar(str1, str2):
    return SequenceMatcher(None, str1, str2).quick_ratio() >= 0.8

    
def _wordcount(sub_list, str1):

    for info in sub_list:
        if isSimilar(str1, info.key):
            info.count += 1
            info.detail.append(str1)
            return
        
    sub_list.append(Info(str1, 1))


def do_str_match(g_data, index, lines, start, end):
    
    p("info: do_str_match(%s, %s) BEGIN" % (start, end))
    
    for i in range(start, end + 1):
        _wordcount(g_data.all_list[index].list, lines[i])

    g_data.appendReady(index)
    
    p("info: do_str_match(%s, %s) DONE" % (start, end))


def do_merge(g_data, sub_list1, sub_list2):
    
    for info1 in sub_list1.list:
        for i in range(len(sub_list2.list)):
#         for info2 in sub_list2:
            if isSimilar(info1.key, (sub_list2.list)[i].key):
                info1.count += (sub_list2.list)[i].count
                (info1.detail).extend((sub_list2.list)[i].detail)
                (sub_list2.list).__delitem__(i)
                break
    
    (sub_list1.list).extend(sub_list2.list)
    
    g_data.appendReadyWithFlag(sub_list1.index)

    
if __name__ == '__main__':

    # 1.读文件
    with open('1000.txt', 'rb') as f:
        lines = f.readlines()
    
    # 2.分段
    nrows = len(lines)
    count = 20
    part = nrows // count
    
    g_data = GlobalData(count)
    
    # 3.分段统计
    threads = []
    for i in range(0, count):
        begin = i * part
        if i != count - 1:
            end = (i + 1) * part - 1
        else:
            end = nrows - 1
        
        t = threading.Thread(target=do_str_match, args=(g_data, i, lines, begin, end))
        threads.append(t)

    print(datetime.now())

    for t in threads:
        t.start()
        
    # 4.周期轮巡
    while True:
        sleep(5)
        print(g_data.getProcess())
        
        # 退出条件：分段统计完成 & 字典合并完成
        if g_data.isFinish():
            print("break!")
            break
        
        # 字典合并
        while True:
            ret, infoList1, infoList2 = g_data.popTop2Ready()
            if not ret:
                break
            
            t = threading.Thread(target=do_merge, args=(g_data, infoList1, infoList2))
            threads.append(t)
            t.start()
          
    for t in threads:
        t.join()

    print(datetime.now())
        
    # 5.排序
    infoList = g_data.getLastInfoList()
    for info in infoList:
        info.detail = sorted(info.detail)
    result = sorted(infoList, key=lambda x:x.count, reverse=True)
    
    print(datetime.now())
    
    # 6.校验总和
    sum1 = 0;
    for info in result:
        print(info)
        sum1 += info.count
    print("info: sum=%s" % sum1)
        
    print("*** OVER ***")

